package de.webcrawler;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.Source;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;

import de.preisrobot.service.helper.HttpClientFactory;

public class Notebooksbilliger {
 
		private static String sourceUrlString = "http://www.notebooksbilliger.de/";

		public static void main(String[] args) throws HttpException, IOException {

		 

			List<String> links = getCategorieLinks(sourceUrlString);
			List<String> links2 = new ArrayList<String>();
			
			for (String string : links) {
				List<String> list = getCategorieLinks(sourceUrlString);
				links2.addAll(list);
			}
			
			
			System.out.println("size "+links2.size());
			
			
			FileOutputStream out = new FileOutputStream("d:\\notebooksbilliger.txt");
			
			for (String string : links2) {
				out.write(string.getBytes());
				
			}
			out.flush();
			out.close();
			
			
			
			
		}

		private static List<String> getCategorieLinks(String sourceUrlString)
				throws IOException, HttpException {
			HttpClient client = HttpClientFactory.getHttpClient();

			GetMethod method = HttpClientFactory.createGETMethod(sourceUrlString);
			client.executeMethod(method);

			try {

				InputStream in = method.getResponseBodyAsStream();
				return parseStream(in);

			} catch (Exception e) {
				e.printStackTrace();

			} finally {

				method.releaseConnection();
			}
			return null;
		}

		private static List<String> parseStream(InputStream in) throws Exception {
			 
			List<String> catUrls = new ArrayList<String>();
			
			Source source = new Source(in);

			List<Element> allElementsByClass = source
					.getAllElementsByClass("categories");

			if(allElementsByClass == null || allElementsByClass.isEmpty())
				return catUrls;
			
			List<Element> elementList = allElementsByClass.get(0).getAllElements(
					"a");

			for (Element element : elementList) {
				String attributeValue = element.getAttributeValue("href");
 
					System.out.println(attributeValue);
					
					catUrls.add(attributeValue);
			 
			}
				return catUrls;
		}
	}
